{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d0e6deaf",
   "metadata": {},
   "source": [
    "# Lesson 4: Using Real-Time Energy Data for Low-Carbon Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3763854d",
   "metadata": {
    "height": 166
   },
   "outputs": [],
   "source": [
    "# Initialize VertexAI\n",
    "from helper import authenticate\n",
    "CREDENTIALS, PROJECT_ID = authenticate()\n",
    "\n",
    "from google.cloud import aiplatform\n",
    "\n",
    "aiplatform.init(project=PROJECT_ID,\n",
    "                credentials=CREDENTIALS,\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26f7f1fa",
   "metadata": {},
   "source": [
    "#### Load the Electricity Maps API for this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fb1a4e6-2ced-4991-b460-d0d62cebf2a6",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "352921a6-c5fa-4976-b517-072fb7e18b05",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "from helper import load_env\n",
    "load_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "068ff8e8-0edf-47fb-90bd-96db9d62511e",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "from helper import load_emaps_api_key\n",
    "API_KEY = load_emaps_api_key()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9db9de9b-8428-4c3c-9b80-78c8451cb79b",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "# Iowa coordinates for this example\n",
    "coordinates = {\"lat\": 41.9216734, \"lon\": -93.3122705}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4039985",
   "metadata": {},
   "source": [
    "> Note: Check here to find more information about [Carbon free energy for Google Cloud regions](https://cloud.google.com/sustainability/region-carbon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4463842e-a2e5-4bd9-a02f-371315e467bd",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "url= f\"https://api.electricitymap.org/v3/carbon-intensity/latest?lon={coordinates['lon']}&lat={coordinates['lat']}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff6b243b-5d82-4936-b84a-642a9ee1e976",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "request = requests.get(url,\n",
    "                       headers={\"auth-token\": API_KEY},)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "117a09f8-7595-473e-a7b9-4446759d6c6f",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "json.loads(request.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d8c9098-6a74-4e21-be60-ab5f674e26aa",
   "metadata": {
    "height": 1101
   },
   "outputs": [],
   "source": [
    "# Dictionary\n",
    "VERTEX_REGIONS = [\n",
    "    {\"id\": \"northamerica-northeast1\", \n",
    "     \"name\": \"Montréal\", \n",
    "     \"lat\": 45.5031824, \"lon\": -73.5698065},\n",
    "    {\"id\": \"northamerica-northeast2\", \n",
    "     \"name\": \"Toronto\", \n",
    "     \"lat\": 43.6534817, \"lon\": -79.3839347},\n",
    "    {\"id\": \"southamerica-east1\", \n",
    "     \"name\": \"São Paulo\", \n",
    "     \"lat\": -23.5506507, \"lon\": -46.6333824},\n",
    "    {\"id\": \"us-central1\", \n",
    "     \"name\": \"Iowa\", \n",
    "     \"lat\": 41.9216734, \"lon\": -93.3122705},\n",
    "    {\"id\": \"us-east1\", \n",
    "     \"name\": \"South Carolina\", \n",
    "     \"lat\": 33.6874388, \"lon\": -80.4363743},\n",
    "    {\"id\": \"us-east4\", \n",
    "     \"name\": \"Northern Virginia\", \n",
    "     \"lat\": -12.548285, \"lon\": 131.017405},\n",
    "    {\"id\": \"us-west1\", \n",
    "     \"name\": \"Oregon\", \n",
    "     \"lat\": 43.9792797, \"lon\": -120.737257},\n",
    "    {\"id\": \"us-west2\", \n",
    "     \"name\": \"Los Angeles\", \n",
    "     \"lat\": 34.0536909, \"lon\": -118.242766},\n",
    "    {\"id\": \"us-west4\", \n",
    "     \"name\": \"Las Vegas\", \n",
    "     \"lat\": 36.1672559, \"lon\": -115.148516},\n",
    "    {\"id\": \"asia-east2\", \n",
    "     \"name\": \"Hong Kong\", \n",
    "     \"lat\": 22.2793278, \"lon\": 114.1628131},\n",
    "    {\"id\": \"asia-northeast1\", \n",
    "     \"name\": \"Tokyo\", \"lat\": 35.6828387, \"lon\": 139.7594549},\n",
    "    {\"id\": \"asia-northeast3\", \n",
    "     \"name\": \"Seoul\", \n",
    "     \"lat\": 37.5666791, \"lon\": 126.9782914},\n",
    "    {\"id\": \"asia-south1\", \n",
    "     \"name\": \"Mumbai\", \n",
    "     \"lat\": 19.0785451, \"lon\": 72.878176},\n",
    "    {\"id\": \"asia-southeast1\", \n",
    "     \"name\": \"Singapore\", \n",
    "     \"lat\": 1.357107, \"lon\": 103.8194992},\n",
    "    {\"id\": \"asia-southeast2\", \n",
    "     \"name\": \"Jakarta\", \n",
    "     \"lat\": -6.1753942, \"lon\": 106.827183},\n",
    "    {\"id\": \"australia-southeast1\", \n",
    "     \"name\": \"Sydney\", \n",
    "     \"lat\": -33.8698439, \"lon\": 151.2082848},\n",
    "    {\"id\": \"europe-west1\", \n",
    "     \"name\": \"Belgium\", \n",
    "     \"lat\": 50.6402809, \"lon\": 4.6667145},\n",
    "    {\"id\": \"europe-west2\", \n",
    "     \"name\": \"London\", \n",
    "     \"lat\": 51.5073219, \"lon\": -0.1276474},\n",
    "    {\"id\": \"europe-west3\", \n",
    "     \"name\": \"Frankfurt\", \n",
    "     \"lat\": 50.1106444, \"lon\": 8.6820917},\n",
    "    {\"id\": \"europe-west4\", \n",
    "     \"name\": \"Netherlands\", \n",
    "     \"lat\": 52.2434979, \"lon\": 5.6343227},\n",
    "    {\"id\": \"europe-west6\", \n",
    "     \"name\": \"Zurich\", \"lat\": 47.3744489, \"lon\": 8.5410422},\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f8b499c-c678-4de0-b483-ae0f7a26d28a",
   "metadata": {
    "height": 370
   },
   "outputs": [],
   "source": [
    "# Returns carbon intensity value of a determinated region\n",
    "def carbon_intensity(VERTEX_REGIONS, API_KEY, region):\n",
    "\n",
    "  region_obj = [r for r \n",
    "                in VERTEX_REGIONS \n",
    "                if region == r[\"id\"]][0]\n",
    "\n",
    "  url= f\"https://api.electricitymap.org/v3/carbon-intensity/latest?lon={region_obj['lon']}&lat={region_obj['lat']}\"\n",
    "\n",
    "  request = requests.get(url,\n",
    "                       headers={\"auth-token\": API_KEY},)\n",
    "\n",
    "  response = json.loads(request.content)\n",
    "\n",
    "  if \"carbonIntensity\" not in response.keys():\n",
    "    raise LookupError(\n",
    "        f\"Carbon intensity data not available \"\n",
    "        f\"from Electricity Maps for region {region}.\"\n",
    "    )\n",
    "\n",
    "  return int(response[\"carbonIntensity\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4cc2856-25b2-4c09-b134-f7ac41b6811f",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "region = 'asia-northeast3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d440a0d7-0bea-4df8-bcd5-6ed9cac53c76",
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "region_obj = [r for r \n",
    "              in VERTEX_REGIONS \n",
    "              if region == r[\"id\"]][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d9ff1d1-1ed6-4caa-9dac-fa106f97b88a",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "region_obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b9817af-ba0d-4833-8f41-58487e7e9da1",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "url= f\"https://api.electricitymap.org/v3/carbon-intensity/latest?lon={region_obj['lon']}&lat={region_obj['lat']}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "300b0847-3d01-4e26-9c8e-07d4f093014e",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "request = requests.get(url,\n",
    "                      headers={\"auth-token\": API_KEY},)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc44c73e-667a-4d0b-9bbc-2704b5a63f0e",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "response = json.loads(request.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0a8fe18-bc6b-481d-a4fe-f3a28682e7ec",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f2b1fb2-245b-4a7b-9367-dc269668c36a",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "int(response[\"carbonIntensity\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9129bfd",
   "metadata": {},
   "source": [
    "* Function cleanest will return the region with lowest carbon intensity value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d1ede78-430a-45f5-a795-ff2eef8bd491",
   "metadata": {
    "height": 370
   },
   "outputs": [],
   "source": [
    "def cleanest(VERTEX_REGIONS, API_KEY):\n",
    "\n",
    "  carbon_intensity_for_regions = []\n",
    "\n",
    "  for region in VERTEX_REGIONS:\n",
    "    try:\n",
    "      carbon_intensity_for_regions.append({\n",
    "        **region,\n",
    "        **{\"carbon_intensity\": carbon_intensity(\n",
    "            VERTEX_REGIONS, \n",
    "            API_KEY, \n",
    "            str(region[\"id\"]))}\n",
    "        })\n",
    "    except LookupError:  # Skip over errors for individual regions.\n",
    "        logging.warning(\n",
    "          f\"Could not get carbon intensity data \"\n",
    "            f\"for region {region['id']}, so it was skipped.\"\n",
    "        )\n",
    "\n",
    "  return min(carbon_intensity_for_regions, \n",
    "             key=lambda x: x[\"carbon_intensity\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "978e2d72-ca2a-4f20-b48d-a3c40d1cc0b2",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "carbon_intensity_for_regions = []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e65fb572",
   "metadata": {},
   "source": [
    "> **Note**: Each double asterix unpacks each dictionaries, meaning that they take out all the key-value pairs of a dictionary, and then adds the key-value pairs into a new dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b6228b9-a545-42ff-957f-6f36bec8ed90",
   "metadata": {
    "height": 149
   },
   "outputs": [],
   "source": [
    "for region in VERTEX_REGIONS:\n",
    "  carbon_intensity_for_regions.append({\n",
    "      **region,\n",
    "      **{\"carbon_intensity\": carbon_intensity(\n",
    "          VERTEX_REGIONS, \n",
    "          API_KEY, \n",
    "          str(region[\"id\"]))}\n",
    "      })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad6a41e5-5512-448e-95b9-ea1c413ca483",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "print(carbon_intensity_for_regions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1848779",
   "metadata": {},
   "source": [
    "> **Note**: This syntax, where two dictionaries are unpacked using double asterisks inside curly braces, means: take all the key-value pairs from each dictionary and combine them into a new dictionary.\n",
    "\n",
    "For example:\n",
    "```\n",
    "d_1 = {'key_1': 1.1}\n",
    "d_2 = {'key_2': 2.2}\n",
    "\n",
    "d_merged = {**d_1, **d_2} \n",
    "print(d_merged)\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f863124",
   "metadata": {},
   "source": [
    "* Find the lowest region with carbon intensity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7bfd1e2-272b-4197-9e62-7bd59c30a9c8",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "min(carbon_intensity_for_regions,\n",
    "    key=lambda x:x['carbon_intensity'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfec7146-68fd-45bc-95f3-9d2cf2f0b165",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "cleanest_region = cleanest(VERTEX_REGIONS, API_KEY)\n",
    "print(cleanest_region)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea12266f-af26-464b-a1ca-30f5220e3b97",
   "metadata": {
    "height": 183
   },
   "outputs": [],
   "source": [
    "# Old carbon intensity region\n",
    "# old_ca = int(response[\"carbonIntensity\"])\n",
    "old_ca = 416\n",
    "\n",
    "# Carbon intensity from the region recently selected\n",
    "# new_ca = int(cleanest_region[\"carbonIntensity\"])\n",
    "new_ca = 36\n",
    "\n",
    "# Compare carbon intensity\n",
    "old_ca/new_ca"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eccd4149",
   "metadata": {},
   "source": [
    "#### Define the Custom Training Job in Vertex AI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8de653ad-2b75-4df9-b6f9-d94e61a5e582",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "REGION = 'northamerica-northeast1'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a87fa00b",
   "metadata": {},
   "source": [
    "* Set up the bucket name and the storage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10637c65-26c2-465c-b7a2-427ee6d0693f",
   "metadata": {
    "height": 217
   },
   "outputs": [],
   "source": [
    "# Create a UUID for the bucket name\n",
    "import random\n",
    "import string\n",
    "\n",
    "def generate_uuid(length: int = 8) -> str:\n",
    "    return \"\".join(\n",
    "        random.choices(\n",
    "            string.ascii_lowercase + string.digits, \n",
    "            k=length))\n",
    "\n",
    "\n",
    "UUID = generate_uuid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4378030c-1805-42e5-ad5c-d25e041f2260",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "bucket_name = f'carbon-course-cleaner-bucket-{UUID}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2691696e",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "# Print\n",
    "bucket_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c977f9f4",
   "metadata": {
    "height": 81
   },
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "storage_client = storage.Client(project = PROJECT_ID,\n",
    "                                credentials = CREDENTIALS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdedcb35-1c49-4d84-bf08-f02ba5da5d08",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "bucket = storage_client.bucket(bucket_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e744b20-2b7a-444a-9e6f-6bbe11bc67ee",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "bucket.create(location=REGION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69933c2f-cec2-4dde-a11b-15003c76dc06",
   "metadata": {
    "height": 166
   },
   "outputs": [],
   "source": [
    "# Custom Training Job\n",
    "job = aiplatform.CustomTrainingJob(\n",
    "    display_name='dlai-course-example-cleaner',\n",
    "    script_path='task.py',\n",
    "    container_uri='us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-14.py310:latest',\n",
    "    staging_bucket=f'gs://{bucket_name}',\n",
    "    location=REGION,)\n",
    "\n",
    "model = job.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eda2f61-3f69-48f9-b25d-ba0db837be4e",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "bucket.delete(force=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3530d205",
   "metadata": {},
   "source": [
    "* Try with 'carbon intensity'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1452bca4-ca83-414c-8454-9155615e357d",
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "url= f\"https://api.electricitymap.org/v3/carbon-intensity/history?lon={cleanest_region['lon']}&lat={cleanest_region['lat']}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b2c9e0a-614d-479c-becb-1b9f4fd66798",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "request = requests.get(url,\n",
    "                       headers={\"auth-token\": API_KEY},)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae3f51dc-b479-4c89-8886-37b8880f3f15",
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "response = json.loads(request.content)\n",
    "response"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
